{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.2"
    },
    "colab": {
      "name": "PyTorch mnist.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "poqOrQo5F3Hj",
        "colab_type": "text"
      },
      "source": [
        "# Quick Start PyTorch - MNIST\n",
        "\n",
        "To run a Code Cell you can click on the `⏯ Run` button in the Navigation Bar above or type `Shift + Enter`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eEAqUCX13sKj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#pip install --force-reinstall torch==1.2.0 torchvision==0.4.0 -f https://download.pytorch.org/whl/torch_stable.html"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UryqtbowF3Ht",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%pylab inline\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.utils.data.dataloader as dataloader\n",
        "import torch.optim as optim\n",
        "\n",
        "from torch.utils.data import TensorDataset\n",
        "from torch.autograd import Variable\n",
        "from torchvision import transforms\n",
        "from torchvision.datasets import MNIST\n",
        "\n",
        "SEED = 1\n",
        "\n",
        "# CUDA?\n",
        "cuda = torch.cuda.is_available()\n",
        "\n",
        "# For reproducibility\n",
        "torch.manual_seed(SEED)\n",
        "\n",
        "if cuda:\n",
        "    torch.cuda.manual_seed(SEED)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9u8WnB6VF3Hy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train = MNIST('./data', train=True, download=True, transform=transforms.Compose([\n",
        "    transforms.ToTensor(), # ToTensor does min-max normalization. \n",
        "]), )\n",
        "\n",
        "test = MNIST('./data', train=False, download=True, transform=transforms.Compose([\n",
        "    transforms.ToTensor(), # ToTensor does min-max normalization. \n",
        "]), )\n",
        "\n",
        "# Create DataLoader\n",
        "dataloader_args = dict(shuffle=True, batch_size=256,num_workers=4, pin_memory=True) if cuda else dict(shuffle=True, batch_size=64)\n",
        "train_loader = dataloader.DataLoader(train, **dataloader_args)\n",
        "test_loader = dataloader.DataLoader(test, **dataloader_args)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bl_QAvj1F3H3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_data = train.train_data\n",
        "train_data = train.transform(train_data.numpy())\n",
        "print(train_data[:, 0, :].shape)\n",
        "for px in train_data[:, 0, :]:\n",
        "  print(px, ' ')\n",
        "\n",
        "print('[Train]')\n",
        "print(' - Numpy Shape:', train.train_data.cpu().numpy().shape)\n",
        "print(' - Tensor Shape:', train.train_data.size())\n",
        "print(' - min:', torch.min(train_data))\n",
        "print(' - max:', torch.max(train_data))\n",
        "print(' - mean:', torch.mean(train_data))\n",
        "print(' - std:', torch.std(train_data))\n",
        "print(' - var:', torch.var(train_data))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CIDtLWm3gSZm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from PIL import Image\n",
        "data = train_data.numpy()[:, 1, :]\n",
        "print(type(data))\n",
        "img = Image.fromarray(data, 'RGB')\n",
        "img.show()\n",
        "#print(img)\n",
        "imshow(data)\n",
        "#print(data)\n",
        "for px in data.flatten():\n",
        "    print(px, ' ', end='')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "07_gZin_F3IL",
        "colab_type": "text"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FutPVey6F3IS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# One hidden Layer NN\n",
        "class Model(nn.Module):\n",
        "    def __init__(self):\n",
        "        batch_size = 100\n",
        "        super(Model, self).__init__()\n",
        "        self.fc = nn.Linear(784, batch_size)\n",
        "        self.fc2 = nn.Linear(batch_size, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = x.view((-1, 784))\n",
        "        h = F.relu(self.fc(x))\n",
        "        h = self.fc2(h)\n",
        "        return F.log_softmax(h)    \n",
        "    \n",
        "    \n",
        "model = Model()\n",
        "if cuda:\n",
        "    model.cuda() # CUDA!\n",
        "optimizer = optim.Adam(model.parameters(), lr=1e-3)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3CQ14d2fF3IV",
        "colab_type": "text"
      },
      "source": [
        "## Train\n",
        "\n",
        "Training time:\n",
        "\n",
        "- CPU, about 1 minute and 30 seconds\n",
        "- GPU, about 10 seconds"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L6IBAfqdF3IX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "EPOCHS = 10\n",
        "losses = []\n",
        "\n",
        "model.train()\n",
        "for epoch in range(EPOCHS):\n",
        "    for batch_idx, (data, target) in enumerate(train_loader):\n",
        "        # Get Samples\n",
        "        data, target = Variable(data), Variable(target)\n",
        "        \n",
        "        if cuda:\n",
        "            data, target = data.cuda(), target.cuda()\n",
        "        \n",
        "        # Init\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # Predict\n",
        "        y_pred = model(data) \n",
        "\n",
        "        # Calculate loss\n",
        "        loss = F.cross_entropy(y_pred, target)\n",
        "        losses.append(loss.cpu().data)\n",
        "        # Backpropagation\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        \n",
        "        \n",
        "        # Display\n",
        "        if batch_idx % 100 == 1:\n",
        "            print('\\r Train Epoch: {}/{} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
        "                epoch+1,\n",
        "                EPOCHS,\n",
        "                batch_idx * len(data), \n",
        "                len(train_loader.dataset),\n",
        "                100. * batch_idx / len(train_loader), \n",
        "                loss.cpu().data), \n",
        "                end='')\n",
        "    # Eval\n",
        "    evaluate_x = Variable(test_loader.dataset.data.type_as(torch.FloatTensor()))\n",
        "    evaluate_y = Variable(test_loader.dataset.targets)\n",
        "    if cuda:\n",
        "        evaluate_x, evaluate_y = evaluate_x.cuda(), evaluate_y.cuda()\n",
        "\n",
        "    model.eval()\n",
        "    output = model(evaluate_x)\n",
        "    pred = output.data.max(1)[1]\n",
        "    d = pred.eq(evaluate_y.data).cpu()\n",
        "    accuracy = d.sum().item()/d.size().numel()\n",
        "    \n",
        "    print('\\r Train Epoch: {}/{} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}\\t Test Accuracy: {:.4f}%'.format(\n",
        "        epoch+1,\n",
        "        EPOCHS,\n",
        "        len(train_loader.dataset), \n",
        "        len(train_loader.dataset),\n",
        "        100. * batch_idx / len(train_loader), \n",
        "        loss.cpu().data,\n",
        "        accuracy*100,\n",
        "        end=''))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r8N-1G72F3Il",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plot(losses)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J6MIPulGF3It",
        "colab_type": "text"
      },
      "source": [
        "## Evaluate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zW-RbqX5F3It",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "evaluate_x = Variable(test_loader.dataset.data.type_as(torch.FloatTensor()))\n",
        "evaluate_y = Variable(test_loader.dataset.targets)\n",
        "if cuda:\n",
        "    evaluate_x, evaluate_y = evaluate_x.cuda(), evaluate_y.cuda()\n",
        "\n",
        "model.eval()\n",
        "output = model(evaluate_x)\n",
        "pred = output.data.max(1)[1]\n",
        "d = pred.eq(evaluate_y.data).cpu()\n",
        "accuracy = d.sum().item()/d.size().numel()\n",
        "\n",
        "print('Accuracy:', accuracy*100.0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kn7xNEYxzqwB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "npi = evaluate_x.cpu().detach().numpy()\n",
        "npo = output.cpu().detach().numpy()\n",
        "for index in range(1):\n",
        "  print(npi[index])\n",
        "  print(npo[index])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OLSGOG-Vz5CN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VxuZSmx2LjbK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dummy_input = torch.randn(784, device='cuda')\n",
        "torch.onnx.export(model, (dummy_input), \"mnist.onnx\", verbose=True)\n",
        "#                  input_names=input_names, output_names=output_names)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZYSqvJE77p8S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pip install onnx==1.5.0\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "80r-Xl5M7xnT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import onnx\n",
        "model=onnx.load_model('mnist.onnx')\n",
        "print(model)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}